{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install -r requirements.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import argparse\n",
    "import logging\n",
    "import joblib\n",
    "import sys\n",
    "import pandas as pd\n",
    "from sklearn.metrics import mean_absolute_error\n",
    "from sklearn.model_selection import train_test_split\n",
    "from xgboost import XGBRegressor\n",
    "try:\n",
    "    from sklearn.preprocessing import Imputer\n",
    "except ImportError:\n",
    "    from sklearn.impute import SimpleImputer as Imputer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "logging.basicConfig(format='%(message)s')\n",
    "logging.getLogger().setLevel(logging.INFO)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def read_input(file_name, test_size=0.25):\n",
    "    \"\"\"Read input data and split it into train and test.\"\"\"\n",
    "    data = pd.read_csv(file_name)\n",
    "    data.dropna(axis=0, subset=['SalePrice'], inplace=True)\n",
    "\n",
    "    y = data.SalePrice\n",
    "    X = data.drop(['SalePrice'], axis=1).select_dtypes(exclude=['object'])\n",
    "\n",
    "    train_X, test_X, train_y, test_y = train_test_split(X.values,\n",
    "                                                      y.values,\n",
    "                                                      test_size=test_size,\n",
    "                                                      shuffle=False)\n",
    "\n",
    "    imputer = Imputer()\n",
    "    train_X = imputer.fit_transform(train_X)\n",
    "    test_X = imputer.transform(test_X)\n",
    "\n",
    "    return (train_X, train_y), (test_X, test_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_model(train_X,\n",
    "                train_y,\n",
    "                test_X,\n",
    "                test_y,\n",
    "                n_estimators,\n",
    "                learning_rate):\n",
    "    \"\"\"Train the model using XGBRegressor.\"\"\"\n",
    "    model = XGBRegressor(n_estimators=n_estimators, learning_rate=learning_rate)\n",
    "\n",
    "    model.fit(train_X,\n",
    "            train_y,\n",
    "            early_stopping_rounds=40,\n",
    "            eval_set=[(test_X, test_y)])\n",
    "\n",
    "    print(\"Best RMSE on eval: %.2f with %d rounds\",\n",
    "               model.best_score,\n",
    "               model.best_iteration+1)\n",
    "    return model\n",
    "\n",
    "def eval_model(model, test_X, test_y):\n",
    "    \"\"\"Evaluate the model performance.\"\"\"\n",
    "    predictions = model.predict(test_X)\n",
    "    logging.info(\"mean_absolute_error=%.2f\", mean_absolute_error(predictions, test_y))\n",
    "\n",
    "def save_model(model, model_file):\n",
    "    \"\"\"Save XGBoost model for serving.\"\"\"\n",
    "    joblib.dump(model, model_file)\n",
    "    logging.info(\"Model export success: %s\", model_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def gcs_copy(src_path, dst_path):\n",
    "    import subprocess\n",
    "    print(subprocess.run(['gsutil', 'cp', src_path, dst_path], stdout=subprocess.PIPE).stdout[:-1].decode('utf-8'))\n",
    "    #print(subprocess.run(['gsutil', 'ls', dst_path], stdout=subprocess.PIPE).stdout[:-1].decode('utf-8'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "### create GCS bucket for storing model weights\n",
    "GCS_BUCKET = \"gs://fairing-demo\"\n",
    "!gsutil mb {GCS_BUCKET}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class HousingServe(object):\n",
    "    \n",
    "    def __init__(self):\n",
    "        self.train_input = \"ames_dataset/train.csv\"\n",
    "        self.n_estimators = 50\n",
    "        self.learning_rate = 0.1\n",
    "        self.model_file = \"trained_ames_model.dat\"\n",
    "        self.gcs_model_file = \"{}/housing_serve/{}\".format(GCS_BUCKET, self.model_file)\n",
    "        self.model = None\n",
    "\n",
    "    def train(self):\n",
    "        (train_X, train_y), (test_X, test_y) = read_input(self.train_input)\n",
    "        model = train_model(train_X,\n",
    "                          train_y,\n",
    "                          test_X,\n",
    "                          test_y,\n",
    "                          self.n_estimators,\n",
    "                          self.learning_rate)\n",
    "\n",
    "        eval_model(model, test_X, test_y)\n",
    "        save_model(model, self.model_file)\n",
    "        gcs_copy(self.model_file, self.gcs_model_file)\n",
    "        print(\"Model saved to {}\".format(self.gcs_model_file))\n",
    "\n",
    "    def predict(self, X):\n",
    "        \"\"\"Predict using the model for given ndarray.\"\"\"\n",
    "        #return [[1.1, 1.2]]\n",
    "        if not self.model:\n",
    "            gcs_copy(self.gcs_model_file, self.model_file)\n",
    "            self.model = joblib.load(self.model_file)\n",
    "        prediction = self.model.predict(data=X)\n",
    "        return [prediction]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training Locally"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "HousingServe().train()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Fairing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "from kubeflow import fairing\n",
    "\n",
    "# Setting up google container repositories (GCR) for storing output containers\n",
    "# You can use any docker container registry istead of GCR\n",
    "GCP_PROJECT = fairing.cloud.gcp.guess_project_name()\n",
    "DOCKER_REGISTRY = 'gcr.io/{}/fairing-job'.format(GCP_PROJECT)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Build base image for remote training and prediction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "py_version = \".\".join([str(x) for x in sys.version_info[0:3]])\n",
    "base_image = 'gcr.io/{}/fairing-predict-example:latest'.format(GCP_PROJECT)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!docker build --build-arg PY_VERSION={py_version} . -t {base_image}\n",
    "!docker push {base_image}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training in KF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "fairing.config.set_builder('docker', registry=DOCKER_REGISTRY, base_image=base_image)\n",
    "fairing.config.set_deployer('job')\n",
    "remote_train = fairing.config.fn(HousingServe)\n",
    "remote_train()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Deploying model and creating an endpoint in KF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fairing.config.set_deployer('serving', serving_class=\"HousingServe\")\n",
    "create_endpoint = fairing.config.fn(HousingServe)\n",
    "create_endpoint()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Making prediction calls against the endpoint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "(train_X, train_y), (test_X, test_y) = read_input(\"ames_dataset/train.csv\")\n",
    "endpoint.predict_nparray(test_X)\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
